package com.xiaotu.spider.douban.pageprocessor;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import org.apache.commons.lang.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.model.ProfessionalCommentModel;
import com.xiaotu.common.model.TVPubCommentModel;
import com.xiaotu.common.model.TVRecordModel;
import com.xiaotu.common.redis.RedisClientTemplate;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.GsonUtils;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.common.util.TVProcessUtils;
import com.xiaotu.spider.AUrlPageProcessor;

import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

/**
 * 豆瓣爬取类--TV 影评和评论
 */
@Component("DouBanPageProcessor")
@Scope("prototype")
public class DouBanPageProcessor extends AUrlPageProcessor
{
	
	private String start_url = "https://www.baidu.com/";
	
	@Autowired
	private RedisClientTemplate redisClient;
	
   /* @Override
    protected Request getStartRequest(JSONObject target)
    {
        return new Request("https://www.douban.com/search?cat=1002&q=" + KeywordEncode.encode(target.getString
                ("tvName")));
    }*/
    
	@Override
	protected String[] getStartRequestUrls() {
		return new String[]{start_url};
	}
	
    @Override
    public String getPageDataType(Page page)
    {
    	return Constants.PageDataType.SEARCH_LIST;
    }

    @Override
    public void process(Page page)
    {
        Map<String, Object> tvMap = page.getRequest().getExtras();
        this.searchListProcess(page, tvMap);
            
    }

    @SuppressWarnings("unused")
    private void searchListProcess(Page page, Map<String, Object> tvMap)
    {
    	
		try {
			Map<String, String> targetMap = redisClient.hgetAll(Constants.DataRedisKey.TV_Info.toString());
			Set<String> set =targetMap.keySet();
			for(String str:set){
				JSONObject jsonbject = GsonUtils.fromJson(targetMap.get(str),JSONObject.class);
				tvMap.putAll(jsonbject);
				Connection conn = connent("https://www.douban.com/search?cat=1002&q=" + 
						KeywordEncode.encode(jsonbject.getString("tvName")));
				conn.get();
				Response rs = conn.response();
				String resHtml = rs.body();
				if(StringUtil.isNotEmpty(resHtml)){
					Html htmlDiv = new Html(resHtml);
					List<String> links = htmlDiv.xpath("//div[@class='result-list']//div[@class='title']/h3/a/@href").all();
			        List<String> names = htmlDiv.xpath("//div[@class='result-list']//div[@class='title']/h3/a/text()").all();
			        names = names.stream().map(name -> name.trim()).collect(Collectors.toList());
			        List<String> tvTypeList = htmlDiv.xpath
			                ("//div[@class='result-list']//div[@class='title']/h3/span[1]/text()").all();

			        List<String> infoStrList = htmlDiv.xpath
			                ("//div[@class='result-list']//div[@class='title']//span[@class='subject-cast']/text()").all();
			        List<List<String>> infoArrayList = new ArrayList<List<String>>();
			        List<List<String>> directorList = new ArrayList<List<String>>();
			        for (String infoStr : infoStrList)
			            infoArrayList.add(Arrays.asList(infoStr.replace(" ", "").split("/")));

			        if (links == null || links.size() == 0)
			            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
			        if (links.size() != names.size() || links.size() != tvTypeList.size() || links.size() != infoArrayList.size())
			            throw new SpiderException(SpiderExceptionCode.DataInvalideException);

			        TVProcessUtils.addTVFromSearchList(page, tvMap, names, links, tvTypeList, infoArrayList, infoArrayList, new
			                TVProcessUtils.TVMatcher()
			                {
			                    @Override
			                    protected boolean addTV(Page page, Map<String, Object> tvinfoMap, String name, String tvType,
			                            List<String> actorsList, List<String> directorsList, String link, int idx)
			                    {
			                        if (TVProcessUtils.tvMatched(tvinfoMap, name.trim(), TVProcessUtils.getTVType(tvType
			                                .replaceAll("\\[|\\]", StringUtils.EMPTY)), actorsList, directorsList))
			                        {
			                        	tvRecordProcess(page,link,tvinfoMap); //详情
			                        	proCommentProcess(page,link + "reviews?sort=time", tvinfoMap);//专业影评
			                        	//PageUtils.addTargetRequest(page, link, tvinfoMap, true);// 详情
			                            //PageUtils.addTargetRequest(page, link + "reviews?sort=time", tvinfoMap);// 专业影评
			                            return true;
			                        }
			                        return false;
			                    }
			                }, null);
				}
				//targetList.add(GsonUtils.fromJson(targetMap.get(str),JSONObject.class));
			}
	       // for (int i = 0; i < targetList.size(); i++){
			
		//}
			
		} catch (IOException e) {
			e.printStackTrace();
		}
    	
    }

    private void tvRecordProcess(Page page, String linkUrl, Map<String, Object> tvMap)
    {

    	try {
			Connection conn = connent(linkUrl);
			conn.get();
			Response rs = conn.response();
			String resHtml = rs.body();
			if (StringUtil.isNotEmpty(resHtml)) {
				Html htmlDiv = new Html(resHtml);
				String score = htmlDiv
						.xpath("//div[@class='rating_wrap clearbox']/p[1]/strong/text()|//div[@class='rating_wrap "
								+ "clearbox']/div[@class='rating_self clearfix']/strong/text()")
						.toString();
				String comment = htmlDiv.xpath("//span[@property='v:votes']/text()").toString();
				System.out.println(KeywordEncode.decode(linkUrl));
				linkUrl = KeywordEncode.decode(linkUrl);
				System.out.println("https://movie.douban.com/"
								+ RegexUtils.regexSearch("(?=subject/).*?(?=/&query)",linkUrl)
								+ "/comments?sort=new_score&status=P");

				if (StringUtils.isEmpty(score) && StringUtils.isEmpty(comment))
					throw new SpiderException(SpiderExceptionCode.DataInvalideException);
				page.putField(TVRecordModel.class.getName(),
						new TVRecordModel(tvMap.get("tvid") + "", TVProcessUtils.getPageTVName(tvMap),
								Constants.SiteName.DOUBAN, score, null, null, null, comment, null, null));
				
				commentProcess(page,
						"https://movie.douban.com/"
								+ RegexUtils.regexSearch("(?=subject/).*?(?=/&query)",linkUrl)
								+ "/comments?sort=new_score&status=P",
						tvMap);// 评论链接（热门）
				commentProcess(page,
						"https://movie.douban.com/"
								+ RegexUtils.regexSearch("(?=subject/).*?(?=/&query)",linkUrl)
								+ "/comments?sort=time&status=P",
						tvMap);// 评论链接（最新）
			} 
		} catch (Exception e) {
			
		}
        
    }

    private void proCommentProcess(Page page, String linkUrl, Map<String, Object> tvMap)
    {
    	
    	try {
			Connection conn = connent(linkUrl);
			conn.get();
			Response rs = conn.response();
			String resHtml = rs.body();
			if (StringUtil.isNotEmpty(resHtml)) {
				Html htmlDiv = new Html(resHtml);
				// 获取专业影评
				List<Selectable> proCommList = htmlDiv.xpath("//div[@class='review-list']/div").nodes();

				if (null == proCommList || proCommList.size() < 1)
					throw new SpiderException(SpiderExceptionCode.DataInvalideException);

				List<ProfessionalCommentModel> pcommentList = new ArrayList<ProfessionalCommentModel>();
				for (Selectable procmt : proCommList) {
					String comment = procmt.xpath("//div[@class='short-content']/text()").toString();
					if (StringUtils.isEmpty(comment) || StringUtils.isEmpty(comment.trim()))
						continue;
					String up = null, down = null;
					String upDown = procmt.xpath("//div[@class='action']/a[@class='action-btn up']/span/text()").toString();
					if (StringUtils.isNotEmpty(upDown)) {
						up = StringUtils.isEmpty(upDown) ? upDown : RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, upDown);
					}
					String downs = procmt.xpath("//div[@class='action']/a[@class='action-btn down']/span/text()").toString();
					if (StringUtils.isNotEmpty(downs)) {
						down = StringUtils.isEmpty(downs) ? downs
								: RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, downs);
					}
					pcommentList.add(new ProfessionalCommentModel(tvMap.get("tvid") + "", tvMap.get("tvName") + "",
							procmt.xpath("//header[@class='main-hd']/a[@class='name']/text()").toString(),
							procmt.xpath("//div[@class='main-bd']/h2/a/text()").toString(), comment,
							procmt.xpath("//header[@class='main-hd']/span[@class='main-meta']/text()").toString(), Constants.SiteName.DOUBAN,
							procmt.xpath("//div[@class='main-bd']/h2/a/@href").toString(), up, down, null));
				}
				if (pcommentList.isEmpty())
					throw new SpiderException(SpiderExceptionCode.DataInvalideException);
				page.putField(ProfessionalCommentModel.class.getName(), pcommentList);
			} 
		} catch (Exception e) {
			
		}
        
    }

    private void commentProcess(Page page,String linkUrl, Map<String, Object> tvMap)
    {
    	try {
			Connection conn = connent(linkUrl);
			conn.get();
			Response rs = conn.response();
			String resHtml = rs.body();
			if (StringUtil.isNotEmpty(resHtml)) {
				Html htmlDiv = new Html(resHtml);
				List<String> reviewList = htmlDiv.xpath("//div[@id='comments']/div[@class='comment-item']")
						.all();
				if (reviewList == null || reviewList.isEmpty())
					throw new SpiderException(SpiderExceptionCode.DataInvalideException);

				List<TVPubCommentModel> commentList = new ArrayList<TVPubCommentModel>();
				for (String reviewHtmlStr : reviewList) {
					Html reviewHtml = new Html(reviewHtmlStr);

					String pubtime = reviewHtml.xpath("//span[@class='comment-time']/text()").toString();
					String user = reviewHtml.xpath("//div[@class='comment']/h3/span[2]/a/text()").toString();
					String comment = reviewHtml.xpath("//div[@class='comment']/p/span/text()").toString();
					if (StringUtil.isEmpty(pubtime) || StringUtils.isEmpty(user) || StringUtils.isEmpty(comment))
						continue;
					commentList.add(new TVPubCommentModel(tvMap.get("tvid") + "", tvMap.get("tvName") + "",
							Constants.SiteName.DOUBAN, user, comment, pubtime.trim(), null,
							page.getUrl().toString() + "comments"));
				}

				if (commentList.isEmpty())
					throw new SpiderException(SpiderExceptionCode.DataInvalideException);
				page.putField(TVPubCommentModel.class.getName(), commentList);
			} 
		} catch (Exception e) {
			
		}
    }

    @Override
    protected String getSpiderType()
    {
        return Constants.SpiderTaskType.TV_DOUBAN;
    }
    
    //链接目标url
  	public static Connection connent(String url) {
      	Connection conn = Jsoup.connect(url)    
	      .userAgent("Mozilla/5.0 (Windows NT 6.1; rv:47.0) Gecko/20100101 Firefox/47.0") // http请求的浏览器设置
	      .timeout(5000)   // http连接时长
	      .maxBodySize(0)
	      .method(Connection.Method.GET);
	      //.cookie("cookie", PropertiesUtil.getProperty("DouBanTVNotBrd.site.Cookie"));  	
      	return conn;
  	}
}
